# coding: utf-8

Help = '''
用key=value来检查指定字符串关键字.

This tool is part of Nlu_Regression Tool project.
Here, it can be used as a json/normal string keywords checking tool.

That means : 
using "key=value;" pairs to check certain string (API result).

e,g.

some api returns:
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
{
	rc: 0,
	text: "请问酒店有健身房吗",
	service: "cn.yunzhisheng.hotel.facility",
	code: "QUERY",
	semantic: {
		intent: {
			focus: "POSITION",
			targetType: "facility",
			tag: "GYM"
	}
},
history: "cn.yunzhisheng.hotel.facility",
responseId: "45391f4174a34d3ab436a12c75178336"
}

can be checking by:
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
service=hotel.facility;code=CALL;focus=POSITION;targetType=facility;tag=GYM

in this case, return unmatched result: ['code => CALL']

key=value pairs grammar support:
http://wiki.it.yzs.io:8090/pages/viewpage.action?pageId=9773440

'''


import re
import time
import json
import urllib2


def get_unmatch_list(expected_output, ret):
    return _get_unmatch_result(expected_output, ret)


############## Below, all copy from caseRunner.py ##############

Null = 'none'
NotNull = 'notnull'
JoinRe = '":\[?"?[^"]*'
List_Picker = '"?:\[[^\]]+\]' # 提取group_list列表内容


def _has_unequal_express(check_str):
    if "!=" in check_str or "=!" in check_str:
        return True
    return False

def _replace_all_unequal(check_str):
    return check_str.replace("=!", "!=")

def _is_key_value_check_points(check_str):
    if _is_json_check_points(check_str):
        return False
    if _is_regular_check_points(check_str):
        return False
    return True

def _is_regular_check_points(check_str):
    if check_str.startswith('<RE') and check_str.endswith('/RE>'):
        return True
    return False

def _is_json_check_points(check_str):
    if check_str.startswith('{') and check_str.endswith('}'):
        return True
    return False

def is_group_list(kv_checks):
    ''' 
    entities=[name=阿司匹林&entityType=MEDICINE&&name=头孢&entityType=MEDICINE]
    '''
    if not kv_checks.endswith(']'):
        return False
    if '=' not in kv_checks:
        return False
    v_list = kv_checks.split('=', 1)[1]
    if '=' not in v_list:
        return False
    if not v_list.startswith('['):
        return False
    return True

def match_group_list(kv_checks, actual_str):
    '''
    1. 先用key=[k=v;k=v;&&k=v;k=v]提取actual_str部分.
    2. 如果提取失败, return False
    3. 然后用k=v;k=v;&&k=v;k=v验证提取的部分.
    name=阿司匹林&entityType=MEDICINE 替换"&"为";"
    4. 通过: return True
    '''
    key, group_list = kv_checks.split('=', 1)
    picker = key + List_Picker

    find_parts = re.findall(picker, actual_str)
    # print 'picker:', picker
    # print 'find_parts: ', find_parts
    if find_parts:
        actual_str = ''.join(find_parts)
    else:
        return False
    for checks in group_list.strip('[]').split('&&'):
        checks = checks.replace('&', ';')
        if _kv_unmatch_result(checks, actual_str):
            return False
        # print "inner match : ", _kv_unmatch_result(checks, actual_str)
    return True

def _get_unmatch_result(checks, actual_str):
    '''
    支持的检查点格式: 
    1. key=value1|value2(竖线表示"或");
    2. key!=!value1|value2(有!=时竖线表示"与")
    3. json字符串和json数组;
    4. 已经写好的正则表达式, 需要用<RE key=regular /RE>标签;
    返回不匹配的key,value列表; 列表为空时, 表示所有检查点Pass.
    '''
    checks = checks.strip()

    # 99%是key=value格式标注格式
    if _is_key_value_check_points(checks):
        return _kv_unmatch_result(checks, actual_str)
    # json/json数组格式检查点: diff_log
    if _is_json_check_points(checks): # json
        return _json_unmatch_result(checks, actual_str)
    # 检查点是已经写好的正则表达式 <RE ... /RE>
    if _is_regular_check_points(checks):
        return _re_unmatch_result(checks, actual_str)

def _kv_unmatch_result(check_str, actual_str):
    """
    匹配成功, 返回空列表; 匹配失败, 返回失败的检查点.
    """
    if _has_unequal_express(check_str):
        check_str = _replace_all_unequal(check_str)
    check_map_list = __kv_check_map_list(check_str)

    buffer_list = []
    for check_map in check_map_list:
        unmatch_buffer = []
        for origin_kv, kv_map in check_map.items():
            if is_group_list(origin_kv):
                if not match_group_list(origin_kv, actual_str):
                    unmatch_buffer.append(origin_kv)
                    break
            for key, v_list in kv_map.items():
                if not key.endswith('!'): # positive
                    regular_list = [JoinRe.join([key, v]) for v in v_list]
                    or_count = 0
                    for regular in regular_list:
                        if re.search(regular, actual_str, re.IGNORECASE):
                            break # 支持'或'方式(key=value1|value2), 有一个匹配即可!
                        else:
                            or_count += 1
                    # 如果所有"或"都没有匹配, 那么此检查点失败.
                    if or_count == len(regular_list):
                        unmatch_buffer.append(origin_kv)
                else: # negative
                    regular_list = [JoinRe.join([key.strip("!"), value]) for value in v_list]
                    if not v_list: # v_list为空, 表示只判断key不存在
                        regular_list = [JoinRe.join([key.strip("!")])]
                    
                    for regular in regular_list:
                        if re.search(regular, actual_str, re.IGNORECASE):
                            unmatch_buffer.append(origin_kv)
                        else:
                            continue # 支持'|'(与)(key!=v1|v2), 表示全不匹配.
        if unmatch_buffer:
            buffer_list.append(unmatch_buffer)
        else:
            # 多组检查的匹配一组即可, 默认双竖线为"或", 需要清空unmatch_list
            return []
    # 如果召回, 但未通过, 那么只显示未通过的一组检查点.
    for buff in buffer_list:
        if 'service=' not in ';'.join(buff):
            return buff
    # 如果全部召回失败, 显示最所有未通过检查点
    unmatch_list = []
    for buff in buffer_list:
        unmatch_list += buff
    return unmatch_list

def _json_unmatch_result(checks, actual_str):
    kv_pairs = _extract_kv_pairs(checks)
    return _kv_unmatch_result(kv_pairs, actual_str)

def _re_unmatch_result(checks, actual_str):
    unmatch_list = []
    for regular in checks[3:-4].strip().split(';'):
        if not re.search(regular, actual_str, re.IGNORECASE):
            unmatch_list.append(regular)
    return unmatch_list

def _get_jkey_list(key_file='conf/keys.conf'):
    jkey_list = []
    for ln in open(key_file):
        ln = ln.strip()
        if not ln:
            continue
        if ln.startswith('#'):
            continue
        if ln not in jkey_list:
            jkey_list.append(ln)
    return jkey_list

def _extract_kv_pairs(checks):
    ''' 从json_str里提前key_value, key是已经整理好的full_regression里的key '''
    kv_pairs = {}
    jkey_list = _get_jkey_list()
    
    for kv_colon_pair in re.findall(r'"[^"]+":"?[^",\[\{:]+"?', checks):
        key, value = kv_colon_pair.split(':')
        key = key.strip('"')
        value = value.strip('"')
        if key == 'text':
            value = 'NotNull'
        if key == 'service':
            value = value.replace('cn.yunzhisheng.', '')
        kv_pair = '%s=%s' % (key, value)
        if key in jkey_list and not kv_pairs.has_key(key):
            kv_pairs[key] = value
    kv_checks = ''
    for k,v in kv_pairs.items():
        kv_checks += '%s=%s;' % (k,v)
    return kv_checks

def _drop_checks_data(case):
    ''' 如果用例是json格式检查点, 那么data, responseId, history都置为空 '''
    if not case.kv_checks.startswith('{'):
        return
    try:
        js_load = json.loads(case.kv_checks)
        js_load['data'] = ''
        js_load['sessionId'] = ''
        js_load['responseId'] = ''
        js_load['history'] = ''
        case.kv_checks = str(js_load)
    except Exception as e:
        print e
        print case.kv_checks
        print '\n\n'
        return

def __kv_check_map_list(key_value_str):
    '''
    1. 用'='分割检查点, 路径的'/'替换为'.{3,}'.
    2. 替换左右两边的特殊字符.
    3. 重新用'.*:.*'合并检查点.
    4. 忽略没有value的检查点(如"content=;")和key = normalHeader的检查点.
    5. 如果检查点key=value对中的value中有空格，去掉的空格.
    6. 加入判断, 不含"="号, 自动忽略检查点
    7. 支持'或', 带"|"的一个检查点对应格式:{'key':['value1', 'value2', ...]}
    8. 检查点中小括号'(' / ')'时, 替换为"\\(", 避免re的unbalanced parenthesis错误.
    9. 新增!=不等于.
    10. 支持非空字段: property=NOTNULL
    11. 如果value是各位数字(如"focusDateIndex":1), 那么单独判断, 更新正则.
    12. 支持key=<value_regular>, value为正则表达式, 语法为尖括号.
    13. 支持key=None, 表示这个key不(应该)存在.
    14. 新增支持双竖线分割多领域.
    15. 增加key=[check_group1&&check_group2]功能, group之间用&&连接, 内部用&连接.
    entities=[name=阿司匹林&entityType=MEDICINE;&&name=头孢&entityType=MEDICINE]
    >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
    示例1: quitDialog=None;service=chat|poem;
    [{  "quitDialog=None": {'quitDialog!' : ['+']},
        "service=chat|poem": {'service', ['chat', 'poem']},
    }]
    示例2: service=audio;tag=灰太狼||service=video;tag=喜羊羊
    [{  "service=audio" : {"service":['audio']},
        "tag=灰太狼"  : {"tag":["灰太狼"]}
     },
        {"service=video" : {"service":['video']},
         "tag=喜羊羊"  : {"tag":["喜羊羊"]},
     },
    ]
    示例3: 单个检查点为group时, 返回为空字典:
    e,g. groups=[name=阿司匹林&type=MEDICINE&&name=头孢&type=MEDICINE]
    '''
    check_map_list = []
    for kv_str in key_value_str.split("||"):
        check_map = {}
        kv_list = kv_str.strip().split(';')
        for kv in kv_list:
            # 先过滤掉一些无效检查点
            if kv == "": continue
            if kv.endswith('='): continue
            if "normalHeader" in kv: continue
            if "keyword" in kv: continue # keyword拆分不是100%正确.
            if "sort" in kv and 'city' in kv: continue
            # weather领域的city=CURRENT_CITY忽略.
            if "city=CURRENT_CITY" in kv: continue
            # 初步清洗检查点
            if '==' in kv:
                kv = kv.replace('==', '=') # 个别检查点标注错误, 使用了双等号
            if kv.startswith('text') and ('(' in kv or ')' in kv):
                kv = kv.replace('(', '\(')
                kv = kv.replace(')', '\)')
            if '=' in kv: # 必须key=value格式才作为有效检查点.
                check_map[kv] = __clean_checks(kv)
            # 单个检查点为group时, 返回为空字典:
            if is_group_list(kv):
                check_map[kv] = {}
        check_map_list.append(check_map)
    return check_map_list

def __clean_checks(kv):
    ''' 进一步清洗和转换kv为字典格式:
        service=chat|poem 转为 {'service', ['chat', 'poem']}
    '''
    splits = kv.strip().split('=')
    k = splits[0]
    # 替换特殊字符
    k = re.sub(r'[*?+()^{}$\\]', '.', k)
    # 空格单独处理,替换为?, 匹配路径/: 如general/intent/type=T
    if " " in k:
        k = k.replace(' ', '\s?')
    if '/' in k:
        k = k.replace('/', '.{3,}')
    # values: 去除'='后的空串, 部分url含'=', 导致'='分割后导致url不完整, 需要重新用'='join连接
    v_list = [v for v in splits[1:] if v.strip()]
    v_list = '='.join(v_list).split('|')
    # 进行封装
    for i in range(len(v_list)):
        v = v_list[i]
        # value是写好的正则表达式, 直接去掉<>标签
        if v.startswith('<') and v.endswith('>'):
            v_list[i] = v.strip().strip('<>')
        else:
            # 替换特殊符号和空格
            v_list[i] = re.sub(r'[*?+()^${}\\]', '.', v)
            if " " in v:
                v_list[i] = v_list[i].replace(' ', '\s?')
            # 支持非空字段检查
            if NotNull == v.lower():
                v_list[i] = '[^"]+'
            # 支持空(即该字段不应该存在), key为!(negative)
            # 同时v_list只有一个值: '+'
            if Null == v.lower():
                k += '!'
                v_list = []

    return {k: v_list}

def __chs_is_in_str(str_maybe_with_chs):
    for ch in str_maybe_with_chs.decode('utf-8'):
        if u'\u4e00' <= ch <= u'\u9fff': return True
    return False

def __get_chinese_char_regular(str_with_CHS):
    """
    搜索关键字被拆分为多个key, 如:　＂韩都衣舍女装毛呢连衣裙＂　返回　＂{"keywords":{"key1":"韩都衣舍","key2":"女装","key3":"毛呢","key4":"连衣裙"}＂
    这时候去掉key1/key2/key3/key4, 直接提取所有汉字用.*连接: 即"keywords"韩.*都.*衣.*舍.*女.*装.*毛.*呢.*连.*衣.*裙"
    """
    chsRegExp = ""
    for chs_char in re.findall(ur'[\u4e00-\u9fff]+', str_with_CHS):
        chsRegExp += chs_char
    chsRegExp = ".*".join(list(chsRegExp))
    return chsRegExp
if __name__ == '__main__':
    check_str = "service=hotel.facility;code=CALL;focus=POSITION;targetType=facility;tag=GYM;actionAble=None"
    actual_str = '{"rc":0,"text":"请问酒店有健身房吗","service":"cn.yunzhisheng.hotel.facility","code":"QUERY","semantic":{"intent":{"focus":"POSITION","targetType":"facility","tag":"GYM"}},"history":"cn.yunzhisheng.hotel.facility","responseId":"42a95c9afb4942dea530bf446e52b232"}'
    print get_unmatch_list(check_str, actual_str) # >> ['code=CALL']

    check_str = "service=none;code=CALL;focus=POSITION;targetType=facility;tag=GYM;notExistKey=None"
    actual_str = '{"rc":0,"text":"请问酒店有健身房吗","service":"cn.yunzhisheng.hotel.facility","code":"QUERY","semantic":{"intent":{"focus":"POSITION","targetType":"facility","tag":"GYM"}},"history":"cn.yunzhisheng.hotel.facility","responseId":"42a95c9afb4942dea530bf446e52b232"}'
    print get_unmatch_list(check_str, actual_str) # >> ['service=none', 'code=CALL']